#!/usr/bin/php
<?php
#######################################################################################################
#Title: Yahoo! Pipe Id crawler
#Description: This script retrieves the list of (Yahoo!) pipes Ids from the category of "popular" pipes
#Author: Carlos Rodriguez
#Creation Date: 09.06.2011
#Last update  : 10.06.2011
#######################################################################################################

include 'util.php';

$matched = array();

if($argc < 3 || $argv[1] != "-c"){
	echo "Usage: pipe_crawler -c number_of_pipes_to_retrieve\n";
	exit;
}

$nbr_pipes = $argv[2];
$nbr_pages = $nbr_pipes / 10;

for($i = 1; $i <= $nbr_pages; $i++){
	#we retrieve a web page that contains a list of popular pipes
	$header = get_web_page("http://pipes.yahoo.com/pipes/pipes.popular?page=$i");
	$content = $header['content'];

	#we search for links (anchors) to the actual pipes to get their Ids
	$pipeCount = preg_match_all('/href=\'\/pipes\/pipe\.info\?_id=([a-zA-Z0-9_]*)\'/', $content, $matched);

	#we obtained the parentethized expresion from the regex above, which corresponds to the Id of the pipe
	#(see manual of preg_match_all() to get more details)
	$ele = $matched[1];

	#in the page we for some reason have duplicate references to the pipe. in this loop we print pipe Ids
	#only once
	for($j = 0; $j < sizeof($ele) - 1; $j++){
		echo $ele[$j] . "\n";
		if($ele[$j] == $ele[$j+1])
			$j++;
	}
}

?>
